import feedparser
import os
import urllib2
from HTMLParser import HTMLParser
#from htmlentitydefs import name2codepoint

class ArticleHTMLParser(HTMLParser):
  indata = False
  memorydata = ''
  def handle_starttag(self, tag, attrs):
    #print 'start tag: ', tag
    if tag == 'title':
      self.indata = True
      print 'indata is ', self.indata
    for attr in attrs:
      #print 'attr: ', attr
      if tag == 'span' and attr[1] == 'midArticle_start':
        self.indata = True
        print 'indata is ', self.indata
      elif tag == 'div' and attr[1] == 'relatedTopicButtons':
        self.indata = False
        print 'indata is ', self.indata
  
  def handle_endtag(self, tag):
    if tag == 'title':
      self.indata = False;
    
  def handle_data(self, data):
    if self.indata == True:
      self.memorydata += data
      #print 'data: ', data

# sources for RSS news
rssSource = { 
    'arts' : 'http://feeds.reuters.com/news/artsculture',
    'health' : 'http://feeds.reuters.com/reuters/healthNews',
    'money' : 'http://feeds.reuters.com/news/wealth',
    'politics' : 'http://feeds.reuters.com/Reuters/PoliticsNews',
    'science' : 'http://feeds.reuters.com/reuters/scienceNews',
    'sports' : 'http://feeds.reuters.com/reuters/sportsNews',
    'technology' : 'http://feeds.reuters.com/reuters/technologyNews' }

parser = ArticleHTMLParser()
entries = []
feeds = []
i = 0
d = ''
for topic in rssSource.keys():
  d = './'+topic+'/'
  if not os.path.exists('./'+topic):
    os.makedirs(d)
  feed = feedparser.parse(rssSource[topic])
  feeds.append(feed)
  for fd in feed['entries']:
    f = open(d+str(i)+'.article', 'w')
    contents = fd['summary'].encode('ascii', 'ignore')
    response = urllib2.urlopen(fd['link'])
    fullart = response.read()
    print 'starting: ', fd['link']
    parser.feed(fullart)
    f.write(parser.memorydata)
    f.close()
    print fd['link']
    i += 1
  

